import pandas as pd
import json

# 读取JSON文件
with open('./post_urls.json', 'r') as file:
    data = json.load(file)

# 将数据转换为DataFrame
df = pd.DataFrame(data)

# 去除重复值
df = df.drop_duplicates()

# 将数据保存回JSON文件
df.to_json('./data_cleaned.json', orient='records', force_ascii=False, indent=4)

# 显示去重后的数据
print(df)
